{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# 第三部分 特征篇（下）"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## 特征选择工具包开发实战\n",
    "\n",
    "- 过滤法：对特征进行某种得分排序，取排名靠前的特征\n",
    "\n",
    "- 包裹法：借助模型，评价不同特征子集的效果，取效果最好的子集\n",
    "\n",
    "- 嵌入法：借助模型自带的特征选择功能实现特征选择，未被选中特征的系数或权重为0"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "首先导入相关的包，包含Filter、Wrapper、Embedded实现需要的接口"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from minepy import MINE\n",
    "from scipy.stats import pearsonr\n",
    "\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.feature_selection import f_classif, f_regression\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "source": [
    "实现特征选择工具类 SelectFeatures"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "def list_diff(list1, list2):\n",
    "    \"\"\"return: 两个list之间的差集\"\"\"\n",
    "    if len(list1) > 0 and len(list2) > 0:\n",
    "        return list(np.setdiff1d(list1, list2))\n",
    "    else:\n",
    "        print('list_diff:len <=0 !!')\n",
    "\n",
    "class SelectFeatures():\n",
    "    '''\n",
    "    X: data, pandas.DataFrame\n",
    "    y: target, pandas.serise 或 nparray\n",
    "    n_features_to_select: 选择特征的数\n",
    "    only_get_index：是否只返回选中特征的索引\n",
    "    '''\n",
    "\n",
    "    \"\"\"\n",
    "    # 首先实现的是初始化和内部方法\n",
    "    \"\"\"\n",
    "    def __init__(self, X, y, n_features_to_select=None, only_get_index=True):\n",
    "        self.cols = X.columns.tolist()\n",
    "        self.X = np.array(X)\n",
    "        self.y = np.array(y)\n",
    "        self.x_index = range(self.X.shape[1])\n",
    "        self.only_get_index = only_get_index\n",
    "        self.n_features_to_select = n_features_to_select\n",
    "        if n_features_to_select is None:\n",
    "            self.n_features_to_select = int(np.ceil(2 / 3 * self.X.shape[1]))\n",
    "            print('self.n_features_to_select: ', self.n_features_to_select)\n",
    "        self.removed = []\n",
    "\n",
    "    def _log(self, index, method):\n",
    "        print('***{}:'.format(method))\n",
    "        print('  remain feature index:\\n  {}'.format(index))\n",
    "        rmvd = list_diff(self.x_index, index)\n",
    "        self.removed += rmvd\n",
    "        print('  removed feature index:\\n  {}\\n'.format(rmvd))\n",
    "\n",
    "    def _return(self, ret, method):\n",
    "        # True代表该特征被选中\n",
    "        index = ret.get_support(indices=True)\n",
    "        self._log(index, method)\n",
    "\n",
    "        if self.only_get_index == True:\n",
    "            return index\n",
    "        else:  #返回筛选之后的X\n",
    "            return ret.transform(self.X)\n",
    "\n",
    "    \"\"\"\n",
    "    # Filter方法使用了SelectKBest, Wrapper方法使用了RFE, Embedded方法使用了SelectFromModel\n",
    "    \"\"\"\n",
    "\n",
    "    # Filter方法\n",
    "    def _by_kbest(self, func, method):\n",
    "        ret = SelectKBest(func,\n",
    "                          k=self.n_features_to_select).fit(self.X, self.y)\n",
    "        return self._return(ret, method)\n",
    "\n",
    "    # Wrapper方法\n",
    "    def _by_RFE(self, mm, method, step=1):\n",
    "        ret = RFE(estimator=mm,\n",
    "                  n_features_to_select=self.n_features_to_select,\n",
    "                  step=step).fit(self.X, self.y)\n",
    "        return self._return(ret, method)\n",
    "\n",
    "    # Embedded方法\n",
    "    def _by_model(self, mm, method):\n",
    "        ret = SelectFromModel(mm).fit(self.X, self.y)\n",
    "        return self._return(ret, method)\n",
    "\n",
    "    \"\"\"\n",
    "    # 具体的Filter方法包含方差、卡方、皮尔森相关系数、最大信息系数和F检验\n",
    "    \"\"\"\n",
    "    # stat\n",
    "    def by_var(self, threshold=0.16):\n",
    "        ret = VarianceThreshold(threshold=threshold).fit(self.X)\n",
    "        return self._return(ret, 'by_var')\n",
    "\n",
    "    def by_chi2(self):\n",
    "        return self._by_kbest(chi2, 'by_chi2')\n",
    "\n",
    "    def by_pearson(self):\n",
    "        ''' 相关系数法 '''\n",
    "        _pp = lambda X, Y: np.array(list(map(lambda x: pearsonr(x, Y), X.T))\n",
    "                                    ).T[0]\n",
    "        return self._by_kbest(_pp, 'by_pearson')\n",
    "\n",
    "    def by_max_info(self):\n",
    "        # or mutual_info_classif\n",
    "        def _mic(x, y):\n",
    "            m = MINE()\n",
    "            m.compute_score(x, y)\n",
    "            return (m.mic(), 0.5)\n",
    "\n",
    "        _pp = lambda X, Y: np.array(list(map(lambda x: _mic(x, Y), X.T))).T[0]\n",
    "        return self._by_kbest(_pp, 'by_max_info')\n",
    "\n",
    "    def by_f_regression(self):\n",
    "        '''\n",
    "        return:\n",
    "            F values of features.\n",
    "            p-values of F-scores.\n",
    "        '''\n",
    "        ret = f_regression(self.X, self.y)\n",
    "        print('Feature importance by f_regression:{}'.format(ret))\n",
    "        return ret\n",
    "\n",
    "    def by_f_classif(self):\n",
    "        ret = f_classif(self.X, self.y)\n",
    "        print('Feature importance by f_regression:{}'.format(ret))\n",
    "        return ret\n",
    "\n",
    "    \"\"\"\n",
    "    # 具体的Wrapper方法包含逻辑回归和支持向量机\n",
    "    \"\"\"\n",
    "\n",
    "    def by_RFE_lr(self, args=None):\n",
    "        return self._by_RFE(LogisticRegression(), 'by_REF_lr')\n",
    "\n",
    "    def by_RFE_svm(self, args=None):\n",
    "        return self._by_RFE(LinearSVC(), 'by_REF_svm')\n",
    "\n",
    "    \"\"\"\n",
    "    # 具体的Embedded方法包含GBDT、随机森林、极端随机树、逻辑回归和支持向量机\n",
    "    \"\"\"\n",
    "    \n",
    "    def by_gbdt(self):\n",
    "        return self._by_model(GradientBoostingClassifier(), 'by_gbdt')\n",
    "\n",
    "    def by_rf(self):\n",
    "        return self._by_model(RandomForestClassifier(), 'by_rf')\n",
    "\n",
    "    def by_et(self):\n",
    "        return self._by_model(ExtraTreesClassifier(), 'by_et')\n",
    "\n",
    "    def by_lr(self, C=0.1):\n",
    "        return self._by_model(\n",
    "            LogisticRegression(penalty='l1', C=C, solver='liblinear'), 'by_lr')\n",
    "\n",
    "    def by_svm(self, C=0.01):\n",
    "        return self._by_model(LinearSVC(penalty='l1', C=C, dual=False),\n",
    "                              'by_svm')\n",
    "\n",
    "    \"\"\"\n",
    "    # 示例中调用了10种特征选择方法，最后投票得出最终的特征选择结果\n",
    "    \"\"\"\n",
    "\n",
    "    # 演示示例\n",
    "    def example_10_methods(self):\n",
    "        name = [\n",
    "            'by_var', 'by_max_info', 'by_pearson', 'by_RFE_svm', 'by_RFE_lr',\n",
    "            'by_svm', 'by_lr', 'by_et', 'by_rf', 'by_gbdt'\n",
    "        ]\n",
    "        # {0:col_0, 1:col_1}\n",
    "        map_index_cols = dict(zip(range(len(self.cols)), self.cols))\n",
    "\n",
    "        # 执行特征选择算法\n",
    "        method_dict = {}\n",
    "        method_dict['by_var'] = self.by_var()\n",
    "        method_dict['by_pearson'] = self.by_pearson()\n",
    "        method_dict['by_max_info'] = self.by_max_info()\n",
    "        method_dict['by_RFE_svm'] = self.by_RFE_svm()\n",
    "        method_dict['by_RFE_lr'] = self.by_RFE_lr()\n",
    "        method_dict['by_svm'] = self.by_svm()\n",
    "        method_dict['by_lr'] = self.by_lr()\n",
    "        method_dict['by_et'] = self.by_et()\n",
    "        method_dict['by_rf'] = self.by_rf()\n",
    "        method_dict['by_gbdt'] = self.by_gbdt()\n",
    "\n",
    "        # 打平选中特征的list\n",
    "        selected = [j for i in list(method_dict.values()) for j in i]\n",
    "\n",
    "        # 构建特征被哪些方法选中：0，1 表示\n",
    "        dicts01 = {}\n",
    "        for nm in name:\n",
    "            dicts01[nm] = [\n",
    "                1 if i in list(method_dict[nm]) else 0\n",
    "                for i in range(len(self.cols))\n",
    "            ]\n",
    "\n",
    "        # 构建结果统计用的DataFrame\n",
    "        stat_f = pd.Series(selected).value_counts().reset_index()\n",
    "        stat_f.columns = ['col_idx', 'count']\n",
    "        stat_f['feature'] = stat_f.col_idx.map(map_index_cols)\n",
    "\n",
    "        # 升序排列匹配模型选择方法的值\n",
    "        stat_f.sort_values(by='col_idx', ascending=True, inplace=True)\n",
    "\n",
    "        for i in name:\n",
    "            stat_f[i] = dicts01[i]\n",
    "\n",
    "        # 按照特征被选中个数降序排列, 个数相同的情况下按照idx升序排列\n",
    "        stat_f.sort_values(by=['count', 'col_idx'],\n",
    "                           ascending=[False, True],\n",
    "                           inplace=True)\n",
    "\n",
    "        selected = stat_f['feature'][:self.n_features_to_select].tolist()\n",
    "        print('*' * 10 + 'remains columns:\\n{}'.format(selected))\n",
    "\n",
    "        return selected, stat_f"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 33,
   "outputs": []
  },
  {
   "source": [
    "导入样例数据"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "0                  5.1               3.5                1.4               0.2   \n",
       "1                  4.9               3.0                1.4               0.2   \n",
       "2                  4.7               3.2                1.3               0.2   \n",
       "3                  4.6               3.1                1.5               0.2   \n",
       "4                  5.0               3.6                1.4               0.2   \n",
       "..                 ...               ...                ...               ...   \n",
       "145                6.7               3.0                5.2               2.3   \n",
       "146                6.3               2.5                5.0               1.9   \n",
       "147                6.5               3.0                5.2               2.0   \n",
       "148                6.2               3.4                5.4               2.3   \n",
       "149                5.9               3.0                5.1               1.8   \n",
       "\n",
       "     target  \n",
       "0         0  \n",
       "1         0  \n",
       "2         0  \n",
       "3         0  \n",
       "4         0  \n",
       "..      ...  \n",
       "145       2  \n",
       "146       2  \n",
       "147       2  \n",
       "148       2  \n",
       "149       2  \n",
       "\n",
       "[150 rows x 5 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>sepal length (cm)</th>\n      <th>sepal width (cm)</th>\n      <th>petal length (cm)</th>\n      <th>petal width (cm)</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>5.1</td>\n      <td>3.5</td>\n      <td>1.4</td>\n      <td>0.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4.9</td>\n      <td>3.0</td>\n      <td>1.4</td>\n      <td>0.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>4.7</td>\n      <td>3.2</td>\n      <td>1.3</td>\n      <td>0.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4.6</td>\n      <td>3.1</td>\n      <td>1.5</td>\n      <td>0.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5.0</td>\n      <td>3.6</td>\n      <td>1.4</td>\n      <td>0.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>145</th>\n      <td>6.7</td>\n      <td>3.0</td>\n      <td>5.2</td>\n      <td>2.3</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>146</th>\n      <td>6.3</td>\n      <td>2.5</td>\n      <td>5.0</td>\n      <td>1.9</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>147</th>\n      <td>6.5</td>\n      <td>3.0</td>\n      <td>5.2</td>\n      <td>2.0</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>148</th>\n      <td>6.2</td>\n      <td>3.4</td>\n      <td>5.4</td>\n      <td>2.3</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>149</th>\n      <td>5.9</td>\n      <td>3.0</td>\n      <td>5.1</td>\n      <td>1.8</td>\n      <td>2</td>\n    </tr>\n  </tbody>\n</table>\n<p>150 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 34
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "\n",
    "df = load_iris(as_frame=True)\n",
    "data = df['data']\n",
    "data['target'] = df['target']\n",
    "\n",
    "data"
   ]
  },
  {
   "source": [
    "准备待筛选的特征列"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['sepal length (cm)',\n",
       " 'sepal width (cm)',\n",
       " 'petal length (cm)',\n",
       " 'petal width (cm)']"
      ]
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "x_col = df['feature_names']\n",
    "x_col"
   ]
  },
  {
   "source": [
    "运行特征选择算法包"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "self.n_features_to_select:  3\n"
     ]
    }
   ],
   "source": [
    "sf = SelectFeatures(data[x_col], data['target'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用过滤法选择特征\n",
    "selected, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "***by_var:\n",
      "  remain feature index:\n",
      "  [0 1 2 3]\n",
      "  removed feature index:\n",
      "  []\n",
      "\n",
      "***by_pearson:\n",
      "  remain feature index:\n",
      "  [0 2 3]\n",
      "  removed feature index:\n",
      "  [1]\n",
      "\n",
      "***by_max_info:\n",
      "  remain feature index:\n",
      "  [0 2 3]\n",
      "  removed feature index:\n",
      "  [1]\n",
      "\n",
      "***by_REF_svm:\n",
      "  remain feature index:\n",
      "  [1 2 3]\n",
      "  removed feature index:\n",
      "  [0]\n",
      "\n",
      "***by_REF_lr:\n",
      "  remain feature index:\n",
      "  [1 2 3]\n",
      "  removed feature index:\n",
      "  [0]\n",
      "\n",
      "***by_svm:\n",
      "  remain feature index:\n",
      "  [0 1 2]\n",
      "  removed feature index:\n",
      "  [3]\n",
      "\n",
      "***by_lr:\n",
      "  remain feature index:\n",
      "  [0 1 2]\n",
      "  removed feature index:\n",
      "  [3]\n",
      "\n",
      "***by_et:\n",
      "  remain feature index:\n",
      "  [2 3]\n",
      "  removed feature index:\n",
      "  [0, 1]\n",
      "\n",
      "***by_rf:\n",
      "  remain feature index:\n",
      "  [2 3]\n",
      "  removed feature index:\n",
      "  [0, 1]\n",
      "\n",
      "***by_gbdt:\n",
      "  remain feature index:\n",
      "  [2 3]\n",
      "  removed feature index:\n",
      "  [0, 1]\n",
      "\n",
      "**********remains columns:\n",
      "['petal length (cm)', 'petal width (cm)', 'sepal length (cm)']\n"
     ]
    }
   ],
   "source": [
    "# 10个选择算法集成\n",
    "selected, stat_f = sf.example_10_methods()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['petal length (cm)', 'petal width (cm)', 'sepal length (cm)']"
      ]
     },
     "metadata": {},
     "execution_count": 38
    }
   ],
   "source": [
    "selected"
   ]
  },
  {
   "source": [
    "stat_f记录了各特征选中的情况\n",
    "\n",
    "- col_idx: 特征在原数据索引\n",
    "\n",
    "- count: 特征共被算法选中的次数\n",
    "\n",
    "- feature: 特征名\n",
    "\n",
    "- by_*: 1表示被算法选中，0表示未选中"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   col_idx  count            feature  by_var  by_max_info  by_pearson  \\\n",
       "0        2     10  petal length (cm)       1            1           1   \n",
       "1        3      8   petal width (cm)       1            1           1   \n",
       "3        0      5  sepal length (cm)       1            1           1   \n",
       "2        1      5   sepal width (cm)       1            0           0   \n",
       "\n",
       "   by_RFE_svm  by_RFE_lr  by_svm  by_lr  by_et  by_rf  by_gbdt  \n",
       "0           1          1       1      1      1      1        1  \n",
       "1           1          1       0      0      1      1        1  \n",
       "3           0          0       1      1      0      0        0  \n",
       "2           1          1       1      1      0      0        0  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>col_idx</th>\n      <th>count</th>\n      <th>feature</th>\n      <th>by_var</th>\n      <th>by_max_info</th>\n      <th>by_pearson</th>\n      <th>by_RFE_svm</th>\n      <th>by_RFE_lr</th>\n      <th>by_svm</th>\n      <th>by_lr</th>\n      <th>by_et</th>\n      <th>by_rf</th>\n      <th>by_gbdt</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2</td>\n      <td>10</td>\n      <td>petal length (cm)</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3</td>\n      <td>8</td>\n      <td>petal width (cm)</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>5</td>\n      <td>sepal length (cm)</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>5</td>\n      <td>sepal width (cm)</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 39
    }
   ],
   "source": [
    "stat_f"
   ]
  }
 ]
}